{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from machine_lib import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n"
     ]
    }
   ],
   "source": [
    "s = login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                     id                        description  \\\n",
      "0                assets                     Assets - Total   \n",
      "1           assets_curr             Current Assets - Total   \n",
      "2                  cogs                 Cost of Goods Sold   \n",
      "3               debt_lt             Long-Term Debt - Total   \n",
      "4                equity     Common/Ordinary Equity - Total   \n",
      "..                  ...                                ...   \n",
      "161  income_beforeextra  Income Before Extraordinary Items   \n",
      "162         liabilities                Liabilities - Total   \n",
      "163    liabilities_curr        Current Liabilities - Total   \n",
      "164             revenue                    Revenue - Total   \n",
      "165               sales               Sales/Turnover (Net)   \n",
      "\n",
      "                                               dataset  \\\n",
      "0    {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "1    {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "2    {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "3    {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "4    {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "..                                                 ...   \n",
      "161  {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "162  {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "163  {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "164  {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "165  {'id': 'fundamental6', 'name': 'Company Fundam...   \n",
      "\n",
      "                                         category  \\\n",
      "0    {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "1    {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "2    {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "3    {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "4    {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "..                                            ...   \n",
      "161  {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "162  {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "163  {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "164  {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "165  {'id': 'fundamental', 'name': 'Fundamental'}   \n",
      "\n",
      "                                           subcategory region  delay universe  \\\n",
      "0    {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "1    {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "2    {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "3    {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "4    {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "..                                                 ...    ...    ...      ...   \n",
      "161  {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "162  {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "163  {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "164  {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "165  {'id': 'fundamental-fundamental-data', 'name':...    AMR      1   TOP600   \n",
      "\n",
      "       type  coverage  userCount  alphaCount  pyramidMultiplier themes  \n",
      "0    MATRIX    0.3499          5           8                1.8     []  \n",
      "1    MATRIX    0.2673          1           1                1.8     []  \n",
      "2    MATRIX    0.2640          5          12                1.8     []  \n",
      "3    MATRIX    0.3261          0           0                1.8     []  \n",
      "4    MATRIX    0.2670          3           6                1.8     []  \n",
      "..      ...       ...        ...         ...                ...    ...  \n",
      "161  MATRIX    0.3496          8          25                1.8     []  \n",
      "162  MATRIX    0.3348          3           6                1.8     []  \n",
      "163  MATRIX    0.2673          0           0                1.8     []  \n",
      "164  MATRIX    0.3481          3           6                1.8     []  \n",
      "165  MATRIX    0.2657          3           4                1.8     []  \n",
      "\n",
      "[166 rows x 14 columns]\n"
     ]
    }
   ],
   "source": [
    "df = get_datafields(s, dataset_id = 'fundamental6', region='AMR', universe='TOP600')\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['winsorize(ts_backfill(assets, 120), std=4)', 'winsorize(ts_backfill(assets_curr, 120), std=4)', 'winsorize(ts_backfill(cogs, 120), std=4)']\n"
     ]
    }
   ],
   "source": [
    "# selected_datafields_df = df[\n",
    "#     (df[\"userCount\"] > 0)\n",
    "#     ].sort_values(by=[\"userCount\"], ascending=False)\n",
    "# print(selected_datafields_df)\n",
    "\n",
    "pc_fields = process_datafields(df, \"matrix\")\n",
    "print(pc_fields[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['winsorize(ts_backfill(assets, 120), std=4)', 'ts_rank(winsorize(ts_backfill(assets, 120), std=4), 5)', 'ts_rank(winsorize(ts_backfill(assets, 120), std=4), 22)', 'ts_rank(winsorize(ts_backfill(assets, 120), std=4), 66)', 'ts_rank(winsorize(ts_backfill(assets, 120), std=4), 120)', 'ts_rank(winsorize(ts_backfill(assets, 120), std=4), 240)', 'ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 5)', 'ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 22)', 'ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 66)', 'ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 120)']\n",
      "14276\n"
     ]
    }
   ],
   "source": [
    "first_order = first_order_factory(pc_fields, ts_ops)\n",
    "print(first_order[:10])\n",
    "print(len(first_order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14276\n",
      "[('winsorize(ts_backfill(assets, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)]\n"
     ]
    }
   ],
   "source": [
    "# Pad initial decay with alpha\n",
    "init_decay = 4\n",
    "fo_alpha_list = []\n",
    "for alpha in first_order:\n",
    "    fo_alpha_list.append((alpha, init_decay))\n",
    "print(len(fo_alpha_list))\n",
    "print(fo_alpha_list[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('winsorize(ts_backfill(assets, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_rank(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_zscore(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_delta(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_delta(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_delta(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_delta(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_delta(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_sum(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_sum(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_sum(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_sum(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_sum(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_product(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_product(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_product(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_product(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_product(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_ir(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_ir(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_ir(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_ir(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_ir(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_std_dev(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_std_dev(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_std_dev(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_std_dev(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_std_dev(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_mean(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_mean(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_mean(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_mean(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_mean(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_arg_min(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_arg_min(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_arg_min(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_arg_min(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_arg_min(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_arg_max(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_arg_max(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_arg_max(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_arg_max(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_arg_max(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_min_diff(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_min_diff(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_min_diff(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_min_diff(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_min_diff(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_max_diff(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_max_diff(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_max_diff(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_max_diff(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_max_diff(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_returns(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_returns(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_returns(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_returns(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_returns(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_scale(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_scale(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_scale(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_scale(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_scale(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_skewness(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_skewness(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_skewness(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_skewness(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_skewness(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_kurtosis(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_kurtosis(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_kurtosis(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_kurtosis(winsorize(ts_backfill(assets, 120), std=4), 120)', 4)], [('ts_kurtosis(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('ts_quantile(winsorize(ts_backfill(assets, 120), std=4), 5)', 4), ('ts_quantile(winsorize(ts_backfill(assets, 120), std=4), 22)', 4), ('ts_quantile(winsorize(ts_backfill(assets, 120), std=4), 66)', 4), ('ts_quantile(winsorize(ts_backfill(assets, 120), std=4), 120)', 4), ('ts_quantile(winsorize(ts_backfill(assets, 120), std=4), 240)', 4), ('winsorize(ts_backfill(assets_curr, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(assets_curr, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(assets_curr, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(assets_curr, 120), std=4), 66)', 4)], [('ts_rank(winsorize(ts_backfill(assets_curr, 120), std=4), 120)', 4), ('ts_rank(winsorize(ts_backfill(assets_curr, 120), std=4), 240)', 4), ('ts_zscore(winsorize(ts_backfill(assets_curr, 120), std=4), 5)', 4), ('ts_zscore(winsorize(ts_backfill(assets_curr, 120), std=4), 22)', 4), ('ts_zscore(winsorize(ts_backfill(assets_curr, 120), std=4), 66)', 4), ('ts_zscore(winsorize(ts_backfill(assets_curr, 120), std=4), 120)', 4), ('ts_zscore(winsorize(ts_backfill(assets_curr, 120), std=4), 240)', 4), ('ts_delta(winsorize(ts_backfill(assets_curr, 120), std=4), 5)', 4), ('ts_delta(winsorize(ts_backfill(assets_curr, 120), std=4), 22)', 4), ('ts_delta(winsorize(ts_backfill(assets_curr, 120), std=4), 66)', 4)]]\n",
      "143\n"
     ]
    }
   ],
   "source": [
    "# Load alphas to task pools\n",
    "pools = load_task_pool(fo_alpha_list, 10, 10)\n",
    "print(pools[0])\n",
    "print(len(pools))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 0 task 9 post done\n",
      "pool 0 task 0 simulate done\n",
      "pool 0 task 1 simulate done\n",
      "pool 0 task 2 simulate done\n",
      "pool 0 task 3 simulate done\n",
      "pool 0 task 4 simulate done\n",
      "pool 0 task 5 simulate done\n",
      "pool 0 task 6 simulate done\n",
      "pool 0 task 7 simulate done\n",
      "pool 0 task 8 simulate done\n",
      "pool 0 task 9 simulate done\n",
      "pool 1 task 9 post done\n",
      "pool 1 task 0 simulate done\n",
      "pool 1 task 1 simulate done\n",
      "pool 1 task 2 simulate done\n",
      "pool 1 task 3 simulate done\n",
      "pool 1 task 4 simulate done\n",
      "pool 1 task 5 simulate done\n",
      "pool 1 task 6 simulate done\n",
      "pool 1 task 7 simulate done\n",
      "pool 1 task 8 simulate done\n",
      "pool 1 task 9 simulate done\n",
      "pool 2 task 9 post done\n",
      "pool 2 task 0 simulate done\n",
      "pool 2 task 1 simulate done\n",
      "pool 2 task 2 simulate done\n",
      "pool 2 task 3 simulate done\n",
      "pool 2 task 4 simulate done\n",
      "pool 2 task 5 simulate done\n",
      "pool 2 task 6 simulate done\n",
      "pool 2 task 7 simulate done\n",
      "pool 2 task 8 simulate done\n",
      "pool 2 task 9 simulate done\n",
      "pool 3 task 9 post done\n",
      "pool 3 task 0 simulate done\n",
      "pool 3 task 1 simulate done\n",
      "pool 3 task 2 simulate done\n",
      "pool 3 task 3 simulate done\n",
      "pool 3 task 4 simulate done\n",
      "pool 3 task 5 simulate done\n",
      "pool 3 task 6 simulate done\n",
      "pool 3 task 7 simulate done\n",
      "pool 3 task 8 simulate done\n",
      "pool 3 task 9 simulate done\n",
      "pool 4 task 9 post done\n",
      "pool 4 task 0 simulate done\n",
      "pool 4 task 1 simulate done\n",
      "pool 4 task 2 simulate done\n",
      "pool 4 task 3 simulate done\n",
      "pool 4 task 4 simulate done\n",
      "pool 4 task 5 simulate done\n",
      "pool 4 task 6 simulate done\n",
      "pool 4 task 7 simulate done\n",
      "pool 4 task 8 simulate done\n",
      "pool 4 task 9 simulate done\n",
      "pool 5 task 9 post done\n",
      "pool 5 task 0 simulate done\n",
      "pool 5 task 1 simulate done\n",
      "pool 5 task 2 simulate done\n",
      "pool 5 task 3 simulate done\n",
      "pool 5 task 4 simulate done\n",
      "pool 5 task 5 simulate done\n",
      "pool 5 task 6 simulate done\n",
      "pool 5 task 7 simulate done\n",
      "pool 5 task 8 simulate done\n",
      "pool 5 task 9 simulate done\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 6 task 9 post done\n",
      "pool 6 task 0 simulate done\n",
      "pool 6 task 1 simulate done\n",
      "pool 6 task 2 simulate done\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 7 task 9 post done\n",
      "pool 7 task 0 simulate done\n",
      "pool 7 task 1 simulate done\n",
      "pool 7 task 2 simulate done\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 8 task 9 post done\n",
      "pool 8 task 0 simulate done\n",
      "pool 8 task 1 simulate done\n",
      "pool 8 task 2 simulate done\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 9 task 9 post done\n",
      "pool 9 task 0 simulate done\n",
      "pool 9 task 1 simulate done\n",
      "pool 9 task 2 simulate done\n",
      "pool 9 task 3 simulate done\n",
      "pool 9 task 4 simulate done\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 10 task 9 post done\n",
      "pool 10 task 0 simulate done\n",
      "pool 10 task 1 simulate done\n",
      "pool 10 task 2 simulate done\n",
      "pool 10 task 3 simulate done\n",
      "pool 10 task 4 simulate done\n",
      "pool 10 task 5 simulate done\n",
      "pool 10 task 6 simulate done\n",
      "pool 10 task 7 simulate done\n",
      "pool 11 task 9 post done\n",
      "pool 11 task 0 simulate done\n",
      "pool 11 task 1 simulate done\n",
      "pool 11 task 2 simulate done\n",
      "pool 11 task 3 simulate done\n",
      "pool 11 task 4 simulate done\n",
      "pool 11 task 5 simulate done\n",
      "pool 11 task 6 simulate done\n",
      "pool 11 task 7 simulate done\n",
      "pool 11 task 8 simulate done\n",
      "pool 11 task 9 simulate done\n",
      "pool 12 task 9 post done\n",
      "pool 12 task 0 simulate done\n",
      "pool 12 task 1 simulate done\n",
      "pool 12 task 2 simulate done\n",
      "pool 12 task 3 simulate done\n",
      "pool 12 task 4 simulate done\n",
      "pool 12 task 5 simulate done\n",
      "pool 12 task 6 simulate done\n",
      "pool 12 task 7 simulate done\n",
      "pool 12 task 8 simulate done\n",
      "pool 12 task 9 simulate done\n",
      "pool 13 task 9 post done\n",
      "pool 13 task 0 simulate done\n",
      "pool 13 task 1 simulate done\n",
      "pool 13 task 2 simulate done\n",
      "pool 13 task 3 simulate done\n",
      "pool 13 task 4 simulate done\n",
      "pool 13 task 5 simulate done\n",
      "pool 13 task 6 simulate done\n",
      "pool 13 task 7 simulate done\n",
      "pool 13 task 8 simulate done\n",
      "pool 13 task 9 simulate done\n",
      "pool 14 task 9 post done\n",
      "pool 14 task 0 simulate done\n",
      "pool 14 task 1 simulate done\n",
      "pool 14 task 2 simulate done\n",
      "pool 14 task 3 simulate done\n",
      "pool 14 task 4 simulate done\n",
      "pool 14 task 5 simulate done\n",
      "pool 14 task 6 simulate done\n",
      "pool 14 task 7 simulate done\n",
      "pool 14 task 8 simulate done\n",
      "pool 14 task 9 simulate done\n",
      "pool 15 task 9 post done\n",
      "pool 15 task 0 simulate done\n",
      "pool 15 task 1 simulate done\n",
      "pool 15 task 2 simulate done\n",
      "pool 15 task 3 simulate done\n",
      "pool 15 task 4 simulate done\n",
      "pool 15 task 5 simulate done\n",
      "pool 15 task 6 simulate done\n",
      "pool 15 task 7 simulate done\n",
      "pool 15 task 8 simulate done\n",
      "pool 15 task 9 simulate done\n",
      "pool 16 task 9 post done\n",
      "pool 16 task 0 simulate done\n",
      "pool 16 task 1 simulate done\n",
      "pool 16 task 2 simulate done\n",
      "pool 16 task 3 simulate done\n",
      "pool 16 task 4 simulate done\n",
      "pool 16 task 5 simulate done\n",
      "pool 16 task 6 simulate done\n",
      "pool 16 task 7 simulate done\n",
      "pool 16 task 8 simulate done\n",
      "pool 16 task 9 simulate done\n",
      "pool 17 task 9 post done\n",
      "pool 17 task 0 simulate done\n",
      "pool 17 task 1 simulate done\n",
      "pool 17 task 2 simulate done\n",
      "pool 17 task 3 simulate done\n",
      "pool 17 task 4 simulate done\n",
      "pool 17 task 5 simulate done\n",
      "pool 17 task 6 simulate done\n",
      "pool 17 task 7 simulate done\n",
      "pool 17 task 8 simulate done\n",
      "pool 17 task 9 simulate done\n",
      "pool 18 task 9 post done\n",
      "pool 18 task 0 simulate done\n",
      "pool 18 task 1 simulate done\n",
      "pool 18 task 2 simulate done\n",
      "pool 18 task 3 simulate done\n",
      "pool 18 task 4 simulate done\n",
      "pool 18 task 5 simulate done\n",
      "pool 18 task 6 simulate done\n",
      "pool 18 task 7 simulate done\n",
      "pool 18 task 8 simulate done\n",
      "pool 18 task 9 simulate done\n",
      "pool 19 task 9 post done\n",
      "pool 19 task 0 simulate done\n",
      "pool 19 task 1 simulate done\n",
      "pool 19 task 2 simulate done\n",
      "pool 19 task 3 simulate done\n",
      "pool 19 task 4 simulate done\n",
      "pool 19 task 5 simulate done\n",
      "pool 19 task 6 simulate done\n",
      "pool 19 task 7 simulate done\n",
      "pool 19 task 8 simulate done\n",
      "pool 19 task 9 simulate done\n",
      "pool 20 task 9 post done\n",
      "pool 20 task 0 simulate done\n",
      "pool 20 task 1 simulate done\n",
      "pool 20 task 2 simulate done\n",
      "pool 20 task 3 simulate done\n",
      "pool 20 task 4 simulate done\n",
      "pool 20 task 5 simulate done\n",
      "pool 20 task 6 simulate done\n",
      "pool 20 task 7 simulate done\n",
      "pool 20 task 8 simulate done\n",
      "pool 20 task 9 simulate done\n",
      "pool 21 task 9 post done\n",
      "pool 21 task 0 simulate done\n",
      "pool 21 task 1 simulate done\n",
      "pool 21 task 2 simulate done\n",
      "pool 21 task 3 simulate done\n",
      "pool 21 task 4 simulate done\n",
      "pool 21 task 5 simulate done\n",
      "pool 21 task 6 simulate done\n",
      "pool 21 task 7 simulate done\n",
      "pool 21 task 8 simulate done\n",
      "pool 21 task 9 simulate done\n",
      "pool 22 task 9 post done\n",
      "other\n",
      "pool 22 task 0 simulate done\n"
     ]
    }
   ],
   "source": [
    "# Simulate First Order\n",
    "multi_simulate(pools, \"SUBINDUSTRY\", \"AMR\", \"TOP600\", 23, 150, \"amr_fo_fund6_sub\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_datafields(s, dataset_id='news85', region='USA', universe='TOP3000', delay=1)\n",
    "print(df[df['type'] == \"VECTOR\"][\"id\"].tolist())\n",
    "print(process_datafields(df, \"vector\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
